#! coding:utf-8

import codecs
import pickle as pkl
import numpy as np

from sklearn.decomposition import PCA
from sklearn.neighbors import BallTree

from web.app.represent_feature import VectorGeneration


def construct_search_model(X, nc, pca_model, search_model):
    pca = PCA(n_components=nc)
    new_X = pca.fit_transform(X)
    print(new_X.shape)
    with open(pca_model, 'wb') as f:
        pkl.dump(pca, f)
    tree = BallTree(new_X)
    with open(search_model, 'wb') as f:
        pkl.dump(tree, f)
    # return pca, tree


def convert_to_vectors(data_file):
    data_list = []
    vg = VectorGeneration()
    with codecs.open(data_file, 'r', 'utf-8') as f:
        while True:
            line = f.readline()
            if not line:
                break
            line = line.rstrip()
            if not line:
                continue
            if not line.startswith('\t'):
                items = line.split('\t')
                line = f.readline()
                user_info = '{0} {1} {2}'.format(line[line.find(':') + 1:].strip(), items[4], items[3])
                f.readline()
                line = f.readline()
                report = line[line.find(':') + 1:].strip()
                line = f.readline()
                inquiry = line[line.find(':') + 1:].strip()
                inquiry = inquiry.replace('$$', '##')
                inquiry = inquiry.replace('@@', '$$')
                inquiry = inquiry.replace('\t', '@@')
                f.readline()
                line = f.readline()
                tongue = line[1:-1].split(',')
                data_list.append(vg.generate_feature_vector(user_info, report, inquiry, tongue))
    return np.asarray(data_list)


if __name__ == '__main__':
    # data = np.random.random((100, 20))
    # pca, tree = construct_search_model(data, 0.8, 'pca.pkl', 'search.pkl')
    # test_data = np.random.random((2, 20))
    # res = tree.query(pca.transform(test_data), k=4, return_distance=False)
    # print(res)
    mat = convert_to_vectors(r'/home/ubuntu/data/web/app/data_valid_complete_origin_0911.txt')
    construct_search_model(mat, 0.8, '/home/ubuntu/data/web/app/pca.pkl', '/home/ubuntu/data/web/app/search_model.pkl')
    print(0)
